{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3472eb71",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-09-09T13:27:27.529484Z",
     "iopub.status.busy": "2021-09-09T13:27:27.521003Z",
     "iopub.status.idle": "2021-09-09T13:30:08.332179Z",
     "shell.execute_reply": "2021-09-09T13:30:08.331585Z",
     "shell.execute_reply.started": "2021-09-08T19:32:15.247446Z"
    },
    "papermill": {
     "duration": 160.835046,
     "end_time": "2021-09-09T13:30:08.332342",
     "exception": false,
     "start_time": "2021-09-09T13:27:27.497296",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\bdone\r\n",
      "Solving environment: / \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\bdone\r\n",
      "\r\n",
      "\r\n",
      "==> WARNING: A newer version of conda exists. <==\r\n",
      "  current version: 4.9.2\r\n",
      "  latest version: 4.10.3\r\n",
      "\r\n",
      "Please update conda by running\r\n",
      "\r\n",
      "    $ conda update -n base conda\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "## Package Plan ##\r\n",
      "\r\n",
      "  environment location: /opt/conda\r\n",
      "\r\n",
      "  added / updated specs:\r\n",
      "    - scipy==1.5.3\r\n",
      "\r\n",
      "\r\n",
      "The following packages will be downloaded:\r\n",
      "\r\n",
      "    package                    |            build\r\n",
      "    ---------------------------|-----------------\r\n",
      "    scipy-1.5.3                |   py37h14a347d_0        19.1 MB  conda-forge\r\n",
      "    ------------------------------------------------------------\r\n",
      "                                           Total:        19.1 MB\r\n",
      "\r\n",
      "The following packages will be DOWNGRADED:\r\n",
      "\r\n",
      "  scipy                                1.6.3-py37h29e03ee_0 --> 1.5.3-py37h14a347d_0\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Downloading and Extracting Packages\r\n",
      "scipy-1.5.3          | 19.1 MB   | ##################################### | 100% \r\n",
      "Preparing transaction: / \b\bdone\r\n",
      "Verifying transaction: \\ \b\b| \b\bdone\r\n",
      "Executing transaction: - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\bdone\r\n"
     ]
    }
   ],
   "source": [
    "# Fixing a problem with Skopt (see https://github.com/scikit-optimize/scikit-optimize/issues/981)\n",
    "!conda install scipy=='1.5.3' --y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "84e3debe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:08.875458Z",
     "iopub.status.busy": "2021-09-09T13:30:08.874741Z",
     "iopub.status.idle": "2021-09-09T13:30:16.199160Z",
     "shell.execute_reply": "2021-09-09T13:30:16.198344Z",
     "shell.execute_reply.started": "2021-09-08T19:34:53.985012Z"
    },
    "papermill": {
     "duration": 7.606245,
     "end_time": "2021-09-09T13:30:16.199341",
     "exception": false,
     "start_time": "2021-09-09T13:30:08.593096",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scikit-learn==0.23.2 in /opt/conda/lib/python3.7/site-packages (0.23.2)\r\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (1.0.1)\r\n",
      "Requirement already satisfied: numpy>=1.13.3 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (1.19.5)\r\n",
      "Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (1.5.3)\r\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (2.1.0)\r\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn=='0.23.2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5312bb2f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:16.758274Z",
     "iopub.status.busy": "2021-09-09T13:30:16.757458Z",
     "iopub.status.idle": "2021-09-09T13:30:17.503510Z",
     "shell.execute_reply": "2021-09-09T13:30:17.502406Z",
     "shell.execute_reply.started": "2021-09-08T19:47:28.036713Z"
    },
    "papermill": {
     "duration": 1.022022,
     "end_time": "2021-09-09T13:30:17.503686",
     "exception": false,
     "start_time": "2021-09-09T13:30:16.481664",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importing core libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shutil\n",
    "from time import time\n",
    "import pprint\n",
    "import joblib\n",
    "from functools import partial\n",
    "import gc\n",
    "\n",
    "# Suppressing warnings because of skopt verbosity\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Classifier/Regressor\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "# Model selection\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Metrics\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "# Skopt functions\n",
    "from skopt import BayesSearchCV\n",
    "from skopt.callbacks import DeadlineStopper, DeltaYStopper\n",
    "from skopt.space import Real, Categorical, Integer\n",
    "from skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments\n",
    "from skopt import gp_minimize, forest_minimize, gbrt_minimize, dummy_minimize\n",
    "\n",
    "# Data processing\n",
    "from sklearn.preprocessing import OrdinalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5c152e1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:18.048379Z",
     "iopub.status.busy": "2021-09-09T13:30:18.047560Z",
     "iopub.status.idle": "2021-09-09T13:30:25.104884Z",
     "shell.execute_reply": "2021-09-09T13:30:25.103979Z",
     "shell.execute_reply.started": "2021-09-08T19:35:02.145439Z"
    },
    "papermill": {
     "duration": 7.331111,
     "end_time": "2021-09-09T13:30:25.105031",
     "exception": false,
     "start_time": "2021-09-09T13:30:17.773920",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loading data \n",
    "X_train = pd.read_csv(\"../input/30-days-of-ml/train.csv\")\n",
    "X_test = pd.read_csv(\"../input/30-days-of-ml/test.csv\")\n",
    "\n",
    "# Preparing data as a tabular matrix\n",
    "X_train = X_train.set_index('id')\n",
    "X_test = X_test.set_index('id')\n",
    "\n",
    "# Pointing out categorical features\n",
    "categoricals = [item for item in X_train.columns if 'cat' in item]\n",
    "\n",
    "# Dealing with categorical data using OrdinalEncoder\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "X_train[categoricals] = ordinal_encoder.fit_transform(X_train[categoricals]).astype(int)\n",
    "X_test[categoricals] = ordinal_encoder.transform(X_test[categoricals]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "59696a2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:25.630091Z",
     "iopub.status.busy": "2021-09-09T13:30:25.629258Z",
     "iopub.status.idle": "2021-09-09T13:30:25.631791Z",
     "shell.execute_reply": "2021-09-09T13:30:25.631375Z",
     "shell.execute_reply.started": "2021-09-08T19:35:09.531516Z"
    },
    "papermill": {
     "duration": 0.265536,
     "end_time": "2021-09-09T13:30:25.631904",
     "exception": false,
     "start_time": "2021-09-09T13:30:25.366368",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the scoring function\n",
    "scoring = partial(mean_squared_error, squared=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8678cc03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:26.162271Z",
     "iopub.status.busy": "2021-09-09T13:30:26.161453Z",
     "iopub.status.idle": "2021-09-09T13:30:26.163837Z",
     "shell.execute_reply": "2021-09-09T13:30:26.164203Z",
     "shell.execute_reply.started": "2021-09-08T19:35:09.539007Z"
    },
    "papermill": {
     "duration": 0.266815,
     "end_time": "2021-09-09T13:30:26.164337",
     "exception": false,
     "start_time": "2021-09-09T13:30:25.897522",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the cv strategy\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2d1372bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:26.688032Z",
     "iopub.status.busy": "2021-09-09T13:30:26.687498Z",
     "iopub.status.idle": "2021-09-09T13:30:30.462573Z",
     "shell.execute_reply": "2021-09-09T13:30:30.462064Z",
     "shell.execute_reply.started": "2021-09-08T19:35:09.545792Z"
    },
    "papermill": {
     "duration": 4.039088,
     "end_time": "2021-09-09T13:30:30.462729",
     "exception": false,
     "start_time": "2021-09-09T13:30:26.423641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fdca4468",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:30.994932Z",
     "iopub.status.busy": "2021-09-09T13:30:30.994305Z",
     "iopub.status.idle": "2021-09-09T13:30:31.008257Z",
     "shell.execute_reply": "2021-09-09T13:30:31.007412Z",
     "shell.execute_reply.started": "2021-09-08T19:35:13.207991Z"
    },
    "papermill": {
     "duration": 0.282502,
     "end_time": "2021-09-09T13:30:31.008373",
     "exception": false,
     "start_time": "2021-09-09T13:30:30.725871",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n",
    "    dataframe = dataframe.copy()\n",
    "    labels = dataframe.pop('target')\n",
    "    ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n",
    "    if shuffle:\n",
    "        ds = ds.shuffle(buffer_size=len(dataframe))\n",
    "    ds = ds.batch(batch_size)\n",
    "    return ds\n",
    "\n",
    "tf.keras.utils.get_custom_objects().update({'leaky-relu': tf.keras.layers.Activation(tf.keras.layers.LeakyReLU(alpha=0.2))})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b5e3ac06",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:31.581722Z",
     "iopub.status.busy": "2021-09-09T13:30:31.580099Z",
     "iopub.status.idle": "2021-09-09T13:30:31.582409Z",
     "shell.execute_reply": "2021-09-09T13:30:31.582831Z",
     "shell.execute_reply.started": "2021-09-08T19:35:13.228651Z"
    },
    "papermill": {
     "duration": 0.276683,
     "end_time": "2021-09-09T13:30:31.582966",
     "exception": false,
     "start_time": "2021-09-09T13:30:31.306283",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_model(cat0_dim, cat1_dim, cat2_dim,\n",
    "                 cat3_dim, cat4_dim, cat5_dim, \n",
    "                 cat6_dim, cat7_dim, cat8_dim, cat9_dim,\n",
    "                 layers, layer_1, layer_2, layer_3, layer_4, layer_5, \n",
    "                 activation, dropout, batch_normalization, learning_rate, **others):\n",
    "    \n",
    "    dims = {'cat0': cat0_dim, 'cat1': cat1_dim, 'cat2': cat2_dim, \n",
    "            'cat3': cat3_dim, 'cat4': cat4_dim, 'cat5': cat5_dim,\n",
    "            'cat6': cat6_dim, 'cat7': cat7_dim, 'cat8': cat8_dim, 'cat9': cat9_dim}\n",
    "    \n",
    "    vocab = {h:X_train['cat4'].unique().astype(int) for h in ['cat0', 'cat1', 'cat2', 'cat3', \n",
    "                                                              'cat4', 'cat5', 'cat6', 'cat7', \n",
    "                                                              'cat8', 'cat9']}\n",
    "    \n",
    "    layers = [layer_1, layer_2, layer_3, layer_4, layer_5][:layers]\n",
    "    \n",
    "    feature_columns = list()\n",
    "\n",
    "    for header in ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6',\n",
    "                   'cont7', 'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13']:\n",
    "        feature_columns.append(tf.feature_column.numeric_column(header))\n",
    "\n",
    "    for header in ['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', \n",
    "                   'cat6', 'cat7', 'cat8', 'cat9']:\n",
    "        feature_columns.append(tf.feature_column.embedding_column(\n",
    "            tf.feature_column.categorical_column_with_vocabulary_list(header, vocabulary_list=vocab[header]), \n",
    "            dimension=dims[header]))\n",
    "\n",
    "    feature_layer = tf.keras.layers.DenseFeatures(feature_columns)\n",
    "\n",
    "    network_struct = [feature_layer]\n",
    "\n",
    "    for nodes in layers:\n",
    "        network_struct.append(tf.keras.layers.Dense(nodes, activation=activation))\n",
    "        if batch_normalization is True:\n",
    "            network_struct.append(tf.keras.layers.BatchNormalization())\n",
    "        if dropout > 0:\n",
    "            network_struct.append(tf.keras.layers.Dropout(dropout))    \n",
    "\n",
    "    model = tf.keras.Sequential(network_struct + [tf.keras.layers.Dense(1)])\n",
    "\n",
    "    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
    "                  loss= tf.keras.losses.MeanSquaredError(),\n",
    "                  metrics=['mean_squared_error'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d2e8ffb9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:32.128197Z",
     "iopub.status.busy": "2021-09-09T13:30:32.127376Z",
     "iopub.status.idle": "2021-09-09T13:30:32.129495Z",
     "shell.execute_reply": "2021-09-09T13:30:32.129885Z",
     "shell.execute_reply.started": "2021-09-08T19:35:13.243691Z"
    },
    "papermill": {
     "duration": 0.284468,
     "end_time": "2021-09-09T13:30:32.130020",
     "exception": false,
     "start_time": "2021-09-09T13:30:31.845552",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the search space\n",
    "    \n",
    "space = [Integer(1, 2, name='cat0_dim'),\n",
    "         Integer(1, 2, name='cat1_dim'),\n",
    "         Integer(1, 2, name='cat2_dim'),\n",
    "         Integer(1, 3, name='cat3_dim'),\n",
    "         Integer(1, 3, name='cat4_dim'),\n",
    "         Integer(1, 3, name='cat5_dim'),\n",
    "         Integer(1, 4, name='cat6_dim'),\n",
    "         Integer(1, 4, name='cat7_dim'),\n",
    "         Integer(1, 6, name='cat8_dim'),\n",
    "         Integer(1, 8, name='cat9_dim'),\n",
    "         Integer(1, 5, name='layers'),\n",
    "         Integer(2, 256, name='layer_1'),\n",
    "         Integer(2, 256, name='layer_2'),\n",
    "         Integer(2, 256, name='layer_3'),\n",
    "         Integer(2, 256, name='layer_4'),\n",
    "         Integer(2, 256, name='layer_5'),\n",
    "         Categorical(['relu', 'leaky-relu'], name='activation'),\n",
    "         Real(0.0, 0.5, 'uniform', name='dropout'),\n",
    "         Categorical([True, False], name='batch_normalization'),\n",
    "         Categorical([0.01, 0.005, 0.002, 0.001], name='learning_rate'),\n",
    "         Integer(256, 1024, name='batch_size')\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9427f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The objective function to be minimized\n",
    "def make_objective(model_fn, X, space, cv, scoring, validation=0.2):\n",
    "    # This decorator converts your objective function with named arguments into one that\n",
    "    # accepts a list as argument, while doing the conversion automatically.\n",
    "    @use_named_args(space) \n",
    "    def objective(**params):\n",
    "        \n",
    "        print(\"\\nTesting: \", params)\n",
    "        validation_scores = list()\n",
    "        \n",
    "        for k, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "            val_index = list()\n",
    "            train_examples = len(train_index)\n",
    "            train_examples = int(train_examples * (1 - validation))\n",
    "            train_index, val_index = train_index[:train_examples], train_index[train_examples:]\n",
    "            \n",
    "            start_time = time()\n",
    "            \n",
    "            model = model_fn(**params)\n",
    "            measure_to_monitor = 'val_mean_squared_error'\n",
    "            modality='min'\n",
    "\n",
    "            early_stopping = tf.keras.callbacks.EarlyStopping(monitor=measure_to_monitor, \n",
    "                                                              mode=modality, \n",
    "                                                              patience=5, \n",
    "                                                              verbose=0)\n",
    "\n",
    "            model_checkpoint = tf.keras.callbacks.ModelCheckpoint('best.model', \n",
    "                                               monitor=measure_to_monitor, \n",
    "                                               mode=modality, \n",
    "                                               save_best_only=True, \n",
    "                                               verbose=0)\n",
    "\n",
    "            run = model.fit(df_to_dataset(X_train.iloc[train_index, :], batch_size=params['batch_size']),\n",
    "                            validation_data=df_to_dataset(X_train.iloc[val_index, :], batch_size=1024),\n",
    "                            epochs=1_000,\n",
    "                            callbacks=[model_checkpoint, early_stopping],\n",
    "                            verbose=0)\n",
    "            \n",
    "            end_time = time()\n",
    "            \n",
    "            rounds = np.argmin(run.history['val_mean_squared_error']) + 1\n",
    "            \n",
    "            model = tf.keras.models.load_model('best.model')\n",
    "            shutil.rmtree('best.model')\n",
    "            \n",
    "            test_preds = model.predict(df_to_dataset(X.iloc[test_index, :], shuffle=False, batch_size=1024)).flatten()\n",
    "            test_score = scoring(X.iloc[test_index, :]['target'], test_preds)\n",
    "            print(f\"CV Fold {k+1} rmse:{test_score:0.5f} - {rounds} rounds - it took {end_time-start_time:0.0f} secs\")\n",
    "            validation_scores.append(test_score)\n",
    "            \n",
    "            if len(history[k]) >= 10:\n",
    "                threshold = np.percentile(history[k], q=25)\n",
    "                if test_score > threshold:\n",
    "                    print(f\"Early stopping for under-performing fold: threshold is {threshold:0.5f}\")\n",
    "                    return np.mean(validation_scores)\n",
    "                \n",
    "            history[k].append(test_score)\n",
    "        return np.mean(validation_scores)\n",
    "\n",
    "    return objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "74c05d4d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:32.665341Z",
     "iopub.status.busy": "2021-09-09T13:30:32.664504Z",
     "iopub.status.idle": "2021-09-09T13:30:32.667304Z",
     "shell.execute_reply": "2021-09-09T13:30:32.666889Z",
     "shell.execute_reply.started": "2021-09-08T19:47:33.415542Z"
    },
    "papermill": {
     "duration": 0.277911,
     "end_time": "2021-09-09T13:30:32.667411",
     "exception": false,
     "start_time": "2021-09-09T13:30:32.389500",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def onstep(res):\n",
    "    global counter\n",
    "    x0 = res.x_iters   # List of input points\n",
    "    y0 = res.func_vals # Evaluation of input points\n",
    "    print('Last eval: ', x0[-1], \n",
    "          ' - Score ', y0[-1])\n",
    "    print('Current iter: ', counter, \n",
    "          ' - Best Score ', res.fun, \n",
    "          ' - Best Args: ', res.x)\n",
    "    joblib.dump((x0, y0), 'checkpoint.pkl') # Saving a checkpoint to disk\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e05eb288",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:33.196626Z",
     "iopub.status.busy": "2021-09-09T13:30:33.195864Z",
     "iopub.status.idle": "2021-09-09T13:30:33.198520Z",
     "shell.execute_reply": "2021-09-09T13:30:33.198133Z",
     "shell.execute_reply.started": "2021-09-08T19:47:35.720639Z"
    },
    "papermill": {
     "duration": 0.268331,
     "end_time": "2021-09-09T13:30:33.198653",
     "exception": false,
     "start_time": "2021-09-09T13:30:32.930322",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "objective = make_objective(create_model,\n",
    "                           X_train,\n",
    "                           space=space,\n",
    "                           cv=kf,\n",
    "                           scoring=scoring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb4b8f30",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "history = {i:list() for i in range(5)}\n",
    "used_time = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3a0af277",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T13:30:33.733203Z",
     "iopub.status.busy": "2021-09-09T13:30:33.732351Z",
     "iopub.status.idle": "2021-09-09T16:46:05.868987Z",
     "shell.execute_reply": "2021-09-09T16:46:05.869413Z"
    },
    "papermill": {
     "duration": 11732.408732,
     "end_time": "2021-09-09T16:46:05.869591",
     "exception": false,
     "start_time": "2021-09-09T13:30:33.460859",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 3, 'cat6_dim': 1, 'cat7_dim': 4, 'cat8_dim': 1, 'cat9_dim': 1, 'layers': 5, 'layer_1': 60, 'layer_2': 195, 'layer_3': 232, 'layer_4': 41, 'layer_5': 89, 'activation': 'relu', 'dropout': 0.19639239805041492, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 371}\n",
      "CV Fold 1 rmse:0.73977 - 13 rounds - it took 272 secs\n",
      "CV Fold 2 rmse:0.73682 - 19 rounds - it took 376 secs\n",
      "CV Fold 3 rmse:0.74341 - 6 rounds - it took 173 secs\n",
      "CV Fold 4 rmse:0.73713 - 8 rounds - it took 191 secs\n",
      "CV Fold 5 rmse:0.73633 - 18 rounds - it took 335 secs\n",
      "Last eval:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, 'relu', 0.19639239805041492, False, 0.005, 371]  - Score  0.7386918524594882\n",
      "Current iter:  0  - Best Score  0.7386918524594882  - Best Args:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, 'relu', 0.19639239805041492, False, 0.005, 371]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 3, 'cat6_dim': 4, 'cat7_dim': 1, 'cat8_dim': 2, 'cat9_dim': 4, 'layers': 4, 'layer_1': 149, 'layer_2': 144, 'layer_3': 169, 'layer_4': 34, 'layer_5': 195, 'activation': 'relu', 'dropout': 0.47233445852479206, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 931}\n",
      "CV Fold 1 rmse:0.74028 - 29 rounds - it took 373 secs\n",
      "CV Fold 2 rmse:0.73705 - 29 rounds - it took 412 secs\n",
      "CV Fold 3 rmse:0.74318 - 31 rounds - it took 418 secs\n",
      "CV Fold 4 rmse:0.73745 - 19 rounds - it took 268 secs\n",
      "CV Fold 5 rmse:0.73700 - 21 rounds - it took 314 secs\n",
      "Last eval:  [1, 2, 2, 3, 1, 3, 4, 1, 2, 4, 4, 149, 144, 169, 34, 195, 'relu', 0.47233445852479206, False, 0.005, 931]  - Score  0.7389899546194652\n",
      "Current iter:  1  - Best Score  0.7386918524594882  - Best Args:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, 'relu', 0.19639239805041492, False, 0.005, 371]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 6, 'cat9_dim': 6, 'layers': 1, 'layer_1': 246, 'layer_2': 19, 'layer_3': 81, 'layer_4': 134, 'layer_5': 107, 'activation': 'relu', 'dropout': 0.30653172894206626, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 313}\n",
      "CV Fold 1 rmse:0.74340 - 1 rounds - it took 93 secs\n",
      "CV Fold 2 rmse:0.73788 - 4 rounds - it took 143 secs\n",
      "CV Fold 3 rmse:0.74722 - 1 rounds - it took 91 secs\n",
      "CV Fold 4 rmse:0.73667 - 7 rounds - it took 185 secs\n",
      "CV Fold 5 rmse:0.73712 - 9 rounds - it took 211 secs\n",
      "Last eval:  [1, 2, 1, 1, 3, 1, 1, 1, 6, 6, 1, 246, 19, 81, 134, 107, 'relu', 0.30653172894206626, False, 0.01, 313]  - Score  0.7404569900293361\n",
      "Current iter:  2  - Best Score  0.7386918524594882  - Best Args:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, 'relu', 0.19639239805041492, False, 0.005, 371]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 6, 'cat9_dim': 5, 'layers': 2, 'layer_1': 172, 'layer_2': 86, 'layer_3': 205, 'layer_4': 70, 'layer_5': 8, 'activation': 'relu', 'dropout': 0.10443837804741737, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 399}\n",
      "CV Fold 1 rmse:0.73936 - 24 rounds - it took 433 secs\n",
      "CV Fold 2 rmse:0.73721 - 8 rounds - it took 198 secs\n",
      "CV Fold 3 rmse:0.74308 - 6 rounds - it took 171 secs\n",
      "CV Fold 4 rmse:0.73573 - 11 rounds - it took 253 secs\n",
      "CV Fold 5 rmse:0.73669 - 13 rounds - it took 259 secs\n",
      "Last eval:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]  - Score  0.7384121958367096\n",
      "Current iter:  3  - Best Score  0.7384121958367096  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 1, 'cat7_dim': 2, 'cat8_dim': 6, 'cat9_dim': 2, 'layers': 4, 'layer_1': 194, 'layer_2': 97, 'layer_3': 199, 'layer_4': 96, 'layer_5': 2, 'activation': 'leaky-relu', 'dropout': 0.33682981541789475, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 810}\n",
      "CV Fold 1 rmse:0.74014 - 16 rounds - it took 241 secs\n",
      "CV Fold 2 rmse:0.73676 - 24 rounds - it took 313 secs\n",
      "CV Fold 3 rmse:0.74553 - 7 rounds - it took 145 secs\n",
      "CV Fold 4 rmse:0.73640 - 19 rounds - it took 282 secs\n",
      "CV Fold 5 rmse:0.73664 - 11 rounds - it took 189 secs\n",
      "Last eval:  [1, 2, 1, 2, 2, 2, 1, 2, 6, 2, 4, 194, 97, 199, 96, 2, 'leaky-relu', 0.33682981541789475, False, 0.001, 810]  - Score  0.7390932322191743\n",
      "Current iter:  4  - Best Score  0.7384121958367096  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 4, 'cat9_dim': 3, 'layers': 4, 'layer_1': 132, 'layer_2': 123, 'layer_3': 100, 'layer_4': 64, 'layer_5': 165, 'activation': 'leaky-relu', 'dropout': 0.10841106881377147, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 818}\n",
      "CV Fold 1 rmse:0.73986 - 22 rounds - it took 310 secs\n",
      "CV Fold 2 rmse:0.78573 - 2 rounds - it took 78 secs\n",
      "CV Fold 3 rmse:0.78790 - 1 rounds - it took 62 secs\n",
      "CV Fold 4 rmse:0.78422 - 2 rounds - it took 75 secs\n",
      "CV Fold 5 rmse:0.74630 - 2 rounds - it took 80 secs\n",
      "Last eval:  [2, 1, 2, 2, 2, 1, 3, 3, 4, 3, 4, 132, 123, 100, 64, 165, 'leaky-relu', 0.10841106881377147, False, 0.001, 818]  - Score  0.7688025335774554\n",
      "Current iter:  5  - Best Score  0.7384121958367096  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 4, 'cat9_dim': 1, 'layers': 4, 'layer_1': 4, 'layer_2': 53, 'layer_3': 82, 'layer_4': 34, 'layer_5': 184, 'activation': 'relu', 'dropout': 0.09159568100355843, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 744}\n",
      "CV Fold 1 rmse:0.74026 - 26 rounds - it took 346 secs\n",
      "CV Fold 2 rmse:0.73726 - 25 rounds - it took 325 secs\n",
      "CV Fold 3 rmse:0.75036 - 1 rounds - it took 67 secs\n",
      "CV Fold 4 rmse:0.73638 - 11 rounds - it took 181 secs\n",
      "CV Fold 5 rmse:0.73780 - 11 rounds - it took 181 secs\n",
      "Last eval:  [2, 1, 2, 3, 2, 1, 3, 3, 4, 1, 4, 4, 53, 82, 34, 184, 'relu', 0.09159568100355843, False, 0.01, 744]  - Score  0.7404102945672485\n",
      "Current iter:  6  - Best Score  0.7384121958367096  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 6, 'cat9_dim': 1, 'layers': 1, 'layer_1': 74, 'layer_2': 21, 'layer_3': 97, 'layer_4': 74, 'layer_5': 156, 'activation': 'leaky-relu', 'dropout': 0.11154081632030918, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 516}\n",
      "CV Fold 1 rmse:0.74136 - 7 rounds - it took 153 secs\n",
      "CV Fold 2 rmse:0.73795 - 9 rounds - it took 188 secs\n",
      "CV Fold 3 rmse:0.74388 - 9 rounds - it took 183 secs\n",
      "CV Fold 4 rmse:0.73618 - 16 rounds - it took 251 secs\n",
      "CV Fold 5 rmse:0.73770 - 13 rounds - it took 221 secs\n",
      "Last eval:  [2, 1, 1, 2, 3, 3, 3, 4, 6, 1, 1, 74, 21, 97, 74, 156, 'leaky-relu', 0.11154081632030918, False, 0.005, 516]  - Score  0.7394134636566191\n",
      "Current iter:  7  - Best Score  0.7384121958367096  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 2, 'cat8_dim': 4, 'cat9_dim': 1, 'layers': 1, 'layer_1': 154, 'layer_2': 159, 'layer_3': 151, 'layer_4': 112, 'layer_5': 27, 'activation': 'leaky-relu', 'dropout': 0.3298342060347065, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 484}\n",
      "CV Fold 1 rmse:0.74089 - 8 rounds - it took 167 secs\n",
      "CV Fold 2 rmse:0.73718 - 11 rounds - it took 217 secs\n",
      "CV Fold 3 rmse:0.74357 - 10 rounds - it took 188 secs\n",
      "CV Fold 4 rmse:0.73659 - 10 rounds - it took 194 secs\n",
      "CV Fold 5 rmse:0.73650 - 28 rounds - it took 423 secs\n",
      "Last eval:  [2, 2, 2, 1, 3, 2, 2, 2, 4, 1, 1, 154, 159, 151, 112, 27, 'leaky-relu', 0.3298342060347065, False, 0.01, 484]  - Score  0.7389477135528918\n",
      "Current iter:  8  - Best Score  0.7384121958367096  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, 'relu', 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 1, 'cat8_dim': 1, 'cat9_dim': 8, 'layers': 2, 'layer_1': 20, 'layer_2': 42, 'layer_3': 184, 'layer_4': 209, 'layer_5': 13, 'activation': 'relu', 'dropout': 0.17702333104190512, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 472}\n",
      "CV Fold 1 rmse:0.73956 - 15 rounds - it took 263 secs\n",
      "CV Fold 2 rmse:0.73670 - 14 rounds - it took 272 secs\n",
      "CV Fold 3 rmse:0.74237 - 13 rounds - it took 239 secs\n",
      "CV Fold 4 rmse:0.73544 - 15 rounds - it took 272 secs\n",
      "CV Fold 5 rmse:0.73647 - 11 rounds - it took 215 secs\n",
      "Last eval:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]  - Score  0.7381087316889905\n",
      "Current iter:  9  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "72432"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gp_round = dummy_minimize(func=objective,\n",
    "                          dimensions=space,\n",
    "                          n_calls=10,\n",
    "                          callback=[onstep],\n",
    "                          random_state=0)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3f947315",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T16:46:07.007238Z",
     "iopub.status.busy": "2021-09-09T16:46:06.997036Z",
     "iopub.status.idle": "2021-09-09T18:54:28.522103Z",
     "shell.execute_reply": "2021-09-09T18:54:28.521677Z"
    },
    "papermill": {
     "duration": 7701.809346,
     "end_time": "2021-09-09T18:54:28.522233",
     "exception": false,
     "start_time": "2021-09-09T16:46:06.712887",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Last eval:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]  - Score  0.7381087316889905\n",
      "Current iter:  10  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 2, 'cat7_dim': 4, 'cat8_dim': 4, 'cat9_dim': 2, 'layers': 2, 'layer_1': 140, 'layer_2': 17, 'layer_3': 72, 'layer_4': 85, 'layer_5': 225, 'activation': 'relu', 'dropout': 0.20626493255557954, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 990}\n",
      "CV Fold 1 rmse:0.74059 - 12 rounds - it took 167 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73979\n",
      "Last eval:  [2, 1, 1, 2, 3, 3, 2, 4, 4, 2, 2, 140, 17, 72, 85, 225, 'relu', 0.20626493255557954, False, 0.005, 990]  - Score  0.7405885357711931\n",
      "Current iter:  11  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 4, 'cat8_dim': 4, 'cat9_dim': 7, 'layers': 1, 'layer_1': 227, 'layer_2': 248, 'layer_3': 37, 'layer_4': 182, 'layer_5': 106, 'activation': 'relu', 'dropout': 0.2718721024098219, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 795}\n",
      "CV Fold 1 rmse:0.74146 - 12 rounds - it took 203 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73979\n",
      "Last eval:  [2, 2, 2, 2, 2, 1, 1, 4, 4, 7, 1, 227, 248, 37, 182, 106, 'relu', 0.2718721024098219, True, 0.001, 795]  - Score  0.7414610171336269\n",
      "Current iter:  12  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 1, 'cat9_dim': 1, 'layers': 2, 'layer_1': 210, 'layer_2': 12, 'layer_3': 2, 'layer_4': 65, 'layer_5': 247, 'activation': 'leaky-relu', 'dropout': 0.24715424379204304, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 471}\n",
      "CV Fold 1 rmse:0.73974 - 17 rounds - it took 311 secs\n",
      "CV Fold 2 rmse:0.73694 - 7 rounds - it took 163 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73688\n",
      "Last eval:  [1, 2, 1, 3, 1, 3, 3, 4, 1, 1, 2, 210, 12, 2, 65, 247, 'leaky-relu', 0.24715424379204304, True, 0.01, 471]  - Score  0.7383393552784108\n",
      "Current iter:  13  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 3, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 1, 'cat9_dim': 3, 'layers': 3, 'layer_1': 65, 'layer_2': 203, 'layer_3': 248, 'layer_4': 227, 'layer_5': 157, 'activation': 'leaky-relu', 'dropout': 0.3956789199083344, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 790}\n",
      "CV Fold 1 rmse:0.74037 - 10 rounds - it took 164 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 1, 1, 2, 2, 3, 1, 1, 1, 3, 3, 65, 203, 248, 227, 157, 'leaky-relu', 0.3956789199083344, True, 0.01, 790]  - Score  0.7403709801291456\n",
      "Current iter:  14  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 2, 'cat8_dim': 2, 'cat9_dim': 8, 'layers': 3, 'layer_1': 2, 'layer_2': 62, 'layer_3': 2, 'layer_4': 256, 'layer_5': 2, 'activation': 'relu', 'dropout': 0.0, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 365}\n",
      "CV Fold 1 rmse:0.74218 - 17 rounds - it took 348 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 1, 2, 2, 2, 4, 2, 2, 8, 3, 2, 62, 2, 256, 2, 'relu', 0.0, True, 0.001, 365]  - Score  0.7421797658421662\n",
      "Current iter:  15  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 7, 'layers': 4, 'layer_1': 23, 'layer_2': 91, 'layer_3': 4, 'layer_4': 96, 'layer_5': 226, 'activation': 'leaky-relu', 'dropout': 0.4423058903938788, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 882}\n",
      "CV Fold 1 rmse:0.74990 - 9 rounds - it took 187 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 2, 3, 2, 2, 3, 2, 5, 7, 4, 23, 91, 4, 96, 226, 'leaky-relu', 0.4423058903938788, False, 0.001, 882]  - Score  0.7499015331175786\n",
      "Current iter:  16  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 3, 'cat8_dim': 5, 'cat9_dim': 2, 'layers': 5, 'layer_1': 90, 'layer_2': 115, 'layer_3': 238, 'layer_4': 66, 'layer_5': 214, 'activation': 'relu', 'dropout': 0.5, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 533}\n",
      "CV Fold 1 rmse:0.74782 - 6 rounds - it took 155 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 1, 1, 1, 2, 4, 3, 5, 2, 5, 90, 115, 238, 66, 214, 'relu', 0.5, True, 0.001, 533]  - Score  0.7478183507817558\n",
      "Current iter:  17  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 2, 'cat8_dim': 2, 'cat9_dim': 7, 'layers': 5, 'layer_1': 205, 'layer_2': 208, 'layer_3': 24, 'layer_4': 73, 'layer_5': 224, 'activation': 'leaky-relu', 'dropout': 0.00749188759191858, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 1018}\n",
      "CV Fold 1 rmse:0.74374 - 7 rounds - it took 135 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 1, 2, 3, 2, 2, 2, 2, 7, 5, 205, 208, 24, 73, 224, 'leaky-relu', 0.00749188759191858, False, 0.001, 1018]  - Score  0.7437449208912379\n",
      "Current iter:  18  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 6, 'cat9_dim': 2, 'layers': 3, 'layer_1': 142, 'layer_2': 54, 'layer_3': 76, 'layer_4': 131, 'layer_5': 11, 'activation': 'relu', 'dropout': 0.2515602980769749, 'batch_normalization': True, 'learning_rate': 0.005, 'batch_size': 648}\n",
      "CV Fold 1 rmse:0.73987 - 13 rounds - it took 213 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 2, 1, 3, 2, 3, 3, 6, 2, 3, 142, 54, 76, 131, 11, 'relu', 0.2515602980769749, True, 0.005, 648]  - Score  0.7398732902007836\n",
      "Current iter:  19  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 1, 'cat7_dim': 4, 'cat8_dim': 2, 'cat9_dim': 6, 'layers': 3, 'layer_1': 176, 'layer_2': 121, 'layer_3': 230, 'layer_4': 81, 'layer_5': 41, 'activation': 'leaky-relu', 'dropout': 0.5, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 578}\n",
      "CV Fold 1 rmse:0.74101 - 4 rounds - it took 119 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 2, 3, 3, 3, 1, 4, 2, 6, 3, 176, 121, 230, 81, 41, 'leaky-relu', 0.5, True, 0.01, 578]  - Score  0.7410148110053308\n",
      "Current iter:  20  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 6, 'cat9_dim': 8, 'layers': 5, 'layer_1': 2, 'layer_2': 73, 'layer_3': 243, 'layer_4': 93, 'layer_5': 186, 'activation': 'relu', 'dropout': 0.0, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 281}\n",
      "CV Fold 1 rmse:0.74843 - 7 rounds - it took 216 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 1, 3, 3, 3, 3, 3, 6, 8, 5, 2, 73, 243, 93, 186, 'relu', 0.0, True, 0.01, 281]  - Score  0.7484329550852917\n",
      "Current iter:  21  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 3, 'cat8_dim': 2, 'cat9_dim': 8, 'layers': 3, 'layer_1': 176, 'layer_2': 31, 'layer_3': 234, 'layer_4': 169, 'layer_5': 206, 'activation': 'leaky-relu', 'dropout': 0.5, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 983}\n",
      "CV Fold 1 rmse:0.74006 - 26 rounds - it took 364 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 2, 1, 3, 1, 2, 4, 3, 2, 8, 3, 176, 31, 234, 169, 206, 'leaky-relu', 0.5, True, 0.002, 983]  - Score  0.7400614650229054\n",
      "Current iter:  22  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 2, 'cat5_dim': 3, 'cat6_dim': 2, 'cat7_dim': 3, 'cat8_dim': 1, 'cat9_dim': 4, 'layers': 2, 'layer_1': 38, 'layer_2': 78, 'layer_3': 203, 'layer_4': 76, 'layer_5': 66, 'activation': 'relu', 'dropout': 0.5, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 1009}\n",
      "CV Fold 1 rmse:0.74029 - 19 rounds - it took 264 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 1, 1, 2, 3, 2, 3, 1, 4, 2, 38, 78, 203, 76, 66, 'relu', 0.5, False, 0.01, 1009]  - Score  0.7402918189389062\n",
      "Current iter:  23  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 6, 'cat9_dim': 8, 'layers': 4, 'layer_1': 211, 'layer_2': 158, 'layer_3': 193, 'layer_4': 244, 'layer_5': 66, 'activation': 'leaky-relu', 'dropout': 0.0, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 1003}\n",
      "CV Fold 1 rmse:0.74159 - 10 rounds - it took 156 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 1, 2, 3, 1, 1, 1, 6, 8, 4, 211, 158, 193, 244, 66, 'leaky-relu', 0.0, True, 0.01, 1003]  - Score  0.7415864936342549\n",
      "Current iter:  24  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 1, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 1, 'cat9_dim': 1, 'layers': 4, 'layer_1': 15, 'layer_2': 203, 'layer_3': 47, 'layer_4': 88, 'layer_5': 153, 'activation': 'relu', 'dropout': 0.0, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 276}\n",
      "CV Fold 1 rmse:0.74159 - 6 rounds - it took 199 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 2, 2, 1, 3, 3, 4, 1, 1, 4, 15, 203, 47, 88, 153, 'relu', 0.0, False, 0.001, 276]  - Score  0.7415854401658187\n",
      "Current iter:  25  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 2, 'cat9_dim': 1, 'layers': 4, 'layer_1': 243, 'layer_2': 27, 'layer_3': 48, 'layer_4': 48, 'layer_5': 256, 'activation': 'leaky-relu', 'dropout': 0.06878403012205843, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 486}\n",
      "CV Fold 1 rmse:0.79346 - 1 rounds - it took 75 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 2, 1, 2, 2, 3, 2, 2, 1, 4, 243, 27, 48, 48, 256, 'leaky-relu', 0.06878403012205843, True, 0.001, 486]  - Score  0.7934550757085055\n",
      "Current iter:  26  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 4, 'cat9_dim': 5, 'layers': 1, 'layer_1': 243, 'layer_2': 166, 'layer_3': 208, 'layer_4': 92, 'layer_5': 115, 'activation': 'relu', 'dropout': 0.1287423841481376, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 547}\n",
      "CV Fold 1 rmse:0.74227 - 5 rounds - it took 120 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 2, 1, 3, 2, 3, 3, 4, 5, 1, 243, 166, 208, 92, 115, 'relu', 0.1287423841481376, True, 0.002, 547]  - Score  0.742265870817151\n",
      "Current iter:  27  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 1, 'cat6_dim': 4, 'cat7_dim': 4, 'cat8_dim': 6, 'cat9_dim': 5, 'layers': 5, 'layer_1': 256, 'layer_2': 256, 'layer_3': 2, 'layer_4': 2, 'layer_5': 2, 'activation': 'leaky-relu', 'dropout': 0.5, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 256}\n",
      "CV Fold 1 rmse:0.74362 - 23 rounds - it took 609 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 2, 3, 1, 1, 4, 4, 6, 5, 5, 256, 256, 2, 2, 2, 'leaky-relu', 0.5, False, 0.001, 256]  - Score  0.7436193040707256\n",
      "Current iter:  28  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 4, 'cat8_dim': 4, 'cat9_dim': 4, 'layers': 3, 'layer_1': 21, 'layer_2': 249, 'layer_3': 79, 'layer_4': 130, 'layer_5': 2, 'activation': 'relu', 'dropout': 0.5, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 970}\n",
      "CV Fold 1 rmse:0.75034 - 3 rounds - it took 89 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 2, 3, 2, 2, 4, 4, 4, 4, 3, 21, 249, 79, 130, 2, 'relu', 0.5, False, 0.005, 970]  - Score  0.7503379495061119\n",
      "Current iter:  29  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 1, 'cat8_dim': 5, 'cat9_dim': 1, 'layers': 1, 'layer_1': 109, 'layer_2': 130, 'layer_3': 88, 'layer_4': 153, 'layer_5': 2, 'activation': 'relu', 'dropout': 0.5, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 972}\n",
      "CV Fold 1 rmse:0.74310 - 7 rounds - it took 130 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 1, 1, 3, 3, 2, 2, 1, 5, 1, 1, 109, 130, 88, 153, 2, 'relu', 0.5, False, 0.005, 972]  - Score  0.7430978888307183\n",
      "Current iter:  30  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 1, 'cat8_dim': 6, 'cat9_dim': 3, 'layers': 4, 'layer_1': 165, 'layer_2': 35, 'layer_3': 91, 'layer_4': 20, 'layer_5': 2, 'activation': 'relu', 'dropout': 0.21831217136553738, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 1004}\n",
      "CV Fold 1 rmse:0.74862 - 5 rounds - it took 104 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 2, 3, 1, 2, 2, 1, 6, 3, 4, 165, 35, 91, 20, 2, 'relu', 0.21831217136553738, False, 0.005, 1004]  - Score  0.7486235115847993\n",
      "Current iter:  31  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 1, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 1, 'cat9_dim': 6, 'layers': 1, 'layer_1': 2, 'layer_2': 256, 'layer_3': 250, 'layer_4': 234, 'layer_5': 2, 'activation': 'leaky-relu', 'dropout': 0.3048586384194193, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 933}\n",
      "CV Fold 1 rmse:0.74096 - 40 rounds - it took 617 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 2, 1, 2, 1, 1, 1, 1, 1, 6, 1, 2, 256, 250, 234, 2, 'leaky-relu', 0.3048586384194193, True, 0.002, 933]  - Score  0.7409609896841414\n",
      "Current iter:  32  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 3, 'cat6_dim': 4, 'cat7_dim': 4, 'cat8_dim': 6, 'cat9_dim': 8, 'layers': 1, 'layer_1': 256, 'layer_2': 2, 'layer_3': 2, 'layer_4': 2, 'layer_5': 2, 'activation': 'leaky-relu', 'dropout': 0.5, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 256}\n",
      "CV Fold 1 rmse:0.74027 - 25 rounds - it took 527 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 2, 1, 3, 2, 3, 4, 4, 6, 8, 1, 256, 2, 2, 2, 2, 'leaky-relu', 0.5, True, 0.001, 256]  - Score  0.7402705727264065\n",
      "Current iter:  33  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 1, 'cat9_dim': 8, 'layers': 1, 'layer_1': 256, 'layer_2': 2, 'layer_3': 256, 'layer_4': 2, 'layer_5': 256, 'activation': 'relu', 'dropout': 0.3527191409539243, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 1024}\n",
      "CV Fold 1 rmse:0.74116 - 9 rounds - it took 154 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [2, 2, 2, 3, 1, 1, 1, 1, 1, 8, 1, 256, 2, 256, 2, 256, 'relu', 0.3527191409539243, True, 0.002, 1024]  - Score  0.7411622487179439\n",
      "Current iter:  34  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 2, 'cat7_dim': 3, 'cat8_dim': 1, 'cat9_dim': 1, 'layers': 5, 'layer_1': 2, 'layer_2': 39, 'layer_3': 2, 'layer_4': 152, 'layer_5': 2, 'activation': 'relu', 'dropout': 0.18096947686879977, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 1024}\n",
      "CV Fold 1 rmse:0.74254 - 31 rounds - it took 481 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73975\n",
      "Last eval:  [1, 1, 1, 3, 2, 1, 2, 3, 1, 1, 5, 2, 39, 2, 152, 2, 'relu', 0.18096947686879977, True, 0.01, 1024]  - Score  0.7425366802024488\n",
      "Current iter:  35  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 4, 'cat9_dim': 4, 'layers': 2, 'layer_1': 256, 'layer_2': 255, 'layer_3': 42, 'layer_4': 254, 'layer_5': 2, 'activation': 'relu', 'dropout': 0.47094816043558835, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 900}\n",
      "CV Fold 1 rmse:0.73947 - 23 rounds - it took 299 secs\n",
      "CV Fold 2 rmse:0.73763 - 4 rounds - it took 99 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73688\n",
      "Last eval:  [2, 2, 2, 1, 2, 2, 1, 1, 4, 4, 2, 256, 255, 42, 254, 2, 'relu', 0.47094816043558835, False, 0.01, 900]  - Score  0.7385495217271978\n",
      "Current iter:  36  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 1, 'cat8_dim': 4, 'cat9_dim': 3, 'layers': 2, 'layer_1': 216, 'layer_2': 242, 'layer_3': 84, 'layer_4': 231, 'layer_5': 175, 'activation': 'leaky-relu', 'dropout': 0.4951650440156478, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 778}\n",
      "CV Fold 1 rmse:0.74088 - 9 rounds - it took 149 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73970\n",
      "Last eval:  [2, 2, 2, 3, 3, 3, 3, 1, 4, 3, 2, 216, 242, 84, 231, 175, 'leaky-relu', 0.4951650440156478, False, 0.005, 778]  - Score  0.7408789939363433\n",
      "Current iter:  37  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 4, 'cat7_dim': 2, 'cat8_dim': 1, 'cat9_dim': 6, 'layers': 1, 'layer_1': 144, 'layer_2': 65, 'layer_3': 123, 'layer_4': 197, 'layer_5': 93, 'activation': 'relu', 'dropout': 0.22691990548052104, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 613}\n",
      "CV Fold 1 rmse:0.74051 - 14 rounds - it took 229 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73970\n",
      "Last eval:  [2, 2, 1, 2, 3, 3, 4, 2, 1, 6, 1, 144, 65, 123, 197, 93, 'relu', 0.22691990548052104, True, 0.002, 613]  - Score  0.7405119462362202\n",
      "Current iter:  38  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 1, 'cat6_dim': 4, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 2, 'layers': 4, 'layer_1': 21, 'layer_2': 73, 'layer_3': 185, 'layer_4': 230, 'layer_5': 2, 'activation': 'leaky-relu', 'dropout': 0.23745052139751802, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 624}\n",
      "CV Fold 1 rmse:0.74068 - 13 rounds - it took 228 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73970\n",
      "Last eval:  [1, 1, 2, 1, 1, 1, 4, 2, 5, 2, 4, 21, 73, 185, 230, 2, 'leaky-relu', 0.23745052139751802, False, 0.005, 624]  - Score  0.7406761819398942\n",
      "Current iter:  39  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 6, 'cat9_dim': 8, 'layers': 4, 'layer_1': 2, 'layer_2': 2, 'layer_3': 2, 'layer_4': 256, 'layer_5': 256, 'activation': 'leaky-relu', 'dropout': 0.5, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 1024}\n",
      "CV Fold 1 rmse:0.74793 - 15 rounds - it took 225 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73970\n",
      "Last eval:  [1, 2, 2, 3, 3, 3, 1, 1, 6, 8, 4, 2, 2, 2, 256, 256, 'leaky-relu', 0.5, False, 0.01, 1024]  - Score  0.7479265228454561\n",
      "Current iter:  40  - Best Score  0.7381087316889905  - Best Args:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, 'relu', 0.17702333104190512, False, 0.01, 472]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "78767"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "\n",
    "gp_round = gp_minimize(func=objective,\n",
    "                       x0=x0,              # already examined values for x\n",
    "                       y0=y0,              # observed values for x0\n",
    "                       dimensions=space,\n",
    "                       acq_func='gp_hedge',\n",
    "                       n_calls=30,\n",
    "                       n_initial_points=0,\n",
    "                       callback=[onstep],\n",
    "                       random_state=0)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f81f00d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T18:54:29.147920Z",
     "iopub.status.busy": "2021-09-09T18:54:29.147314Z",
     "iopub.status.idle": "2021-09-09T18:54:29.152345Z",
     "shell.execute_reply": "2021-09-09T18:54:29.151844Z"
    },
    "papermill": {
     "duration": 0.324329,
     "end_time": "2021-09-09T18:54:29.152462",
     "exception": false,
     "start_time": "2021-09-09T18:54:28.828133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "500b90a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T18:54:29.756207Z",
     "iopub.status.busy": "2021-09-09T18:54:29.755336Z",
     "iopub.status.idle": "2021-09-09T18:54:29.759832Z",
     "shell.execute_reply": "2021-09-09T18:54:29.760261Z"
    },
    "papermill": {
     "duration": 0.308973,
     "end_time": "2021-09-09T18:54:29.760396",
     "exception": false,
     "start_time": "2021-09-09T18:54:29.451423",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.73811\n",
      "Best hyperparameters:\n",
      "cat0_dim                  : 2\n",
      "cat1_dim                  : 2\n",
      "cat2_dim                  : 2\n",
      "cat3_dim                  : 3\n",
      "cat4_dim                  : 2\n",
      "cat5_dim                  : 2\n",
      "cat6_dim                  : 4\n",
      "cat7_dim                  : 1\n",
      "cat8_dim                  : 1\n",
      "cat9_dim                  : 8\n",
      "layers                    : 2\n",
      "layer_1                   : 20\n",
      "layer_2                   : 42\n",
      "layer_3                   : 184\n",
      "layer_4                   : 209\n",
      "layer_5                   : 13\n",
      "activation                : relu\n",
      "dropout                   : 0.17702333104190512\n",
      "batch_normalization       : False\n",
      "learning_rate             : 0.01\n",
      "batch_size                : 472\n"
     ]
    }
   ],
   "source": [
    "print(f\"Best score: {gp_round.fun:0.5f}\")\n",
    "print(\"Best hyperparameters:\")\n",
    "for sp, x in zip(gp_round.space, gp_round.x):\n",
    "    print(f\"{sp.name:25} : {x}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 19631.425975,
   "end_time": "2021-09-09T18:54:32.453099",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-09-09T13:27:21.027124",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
